/*
 *  proxy_checker.cpp
 *  proxy_hunter
 *
 *  Created by Roy Zuo on May 13, 2010.
 *  Copyright 2010 O2 Micro, Inc. All rights reserved.
 *
 */


#include "proxy_checker.hpp"
#include <sstream>
#include <iostream>

#include <boost/algorithm/string.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int.hpp>
#include <boost/random/variate_generator.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/threadpool.hpp>

ProxyChecker::ProxyChecker ( const string& db_file )
: _reader ( 0 )
{
	_reader = new ProxyReader ( db_file );
}

ProxyChecker::~ProxyChecker ()
{
	if ( _reader )
	{
		delete _reader;
		_reader = 0;
	}
}

void ProxyChecker::operator () ()
{
	vector < string > proxies;
	_reader->SelectWithStatus ( 0, proxies );

	string header, content;
	read_test_webpage ( header, content );

	HttpHeaderPairs headers;
	headers["Accept"] = "*/*";
	headers["Connection"] = "close";

	for ( vector< string >::const_iterator iter = proxies.begin(); iter != proxies.end(); iter ++ )
	{
		int seconds = 999999;		
		bool rb = check( header, content, *iter, headers, seconds );
		if ( rb  )
		{
			_reader->Update( *iter, 1, seconds );
		}
		else
		{
			_reader->Update( *iter, 2, seconds );
		}

	}
}

bool ProxyChecker::check ( const string& check_header, const string& check_content, const string& proxy, HttpHeaderPairs headers, int& seconds )
{
	stringstream header_stream;
	stringstream content_stream;

	string server, port;
	_reader->SplitProxy ( proxy, server, port );

	boost::posix_time::ptime start_time = boost::posix_time::second_clock::local_time();

	bool rb = read_webpage ( server, port, HOST_TEST_PROXY, PATH_TEST_PROXY, headers, header_stream, content_stream );
	rb = verify_webpage ( check_header, check_content, header_stream, content_stream );

	boost::posix_time::ptime end_time = boost::posix_time::second_clock::local_time();
	boost::posix_time::time_duration td = end_time - start_time;

	seconds = td.total_milliseconds();
	return rb;
}

void ProxyChecker::handle_check ( const string& check_header, const string& check_content, const string& proxy, HttpHeaderPairs headers, int& seconds )
{
	stringstream header_stream;
	stringstream content_stream;

	string server, port;
	_reader->SplitProxy ( proxy, server, port );

	boost::posix_time::ptime start_time = boost::posix_time::second_clock::local_time();

	bool rb = read_webpage ( server, port, HOST_TEST_PROXY, PATH_TEST_PROXY, headers, header_stream, content_stream );
	rb = verify_webpage ( check_header, check_content, header_stream, content_stream );

	boost::posix_time::ptime end_time = boost::posix_time::second_clock::local_time();
	boost::posix_time::time_duration td = end_time - start_time;

	seconds = td.total_milliseconds();
	
	if ( rb  )
	{
		_reader->Update( proxy, 1, seconds );
	}
	else
	{
		_reader->Update( proxy, 2, seconds );
	}
}

void ProxyChecker::read_test_webpage ( string& header, string& content )
{
	stringstream header_stream, content_stream;
	boost::asio::io_service io_service;

	HttpHeaderPairs headers;
	headers["Accept"] = "*/*";
	headers["Connection"] = "close";

	WebpageHunter hunter ( io_service, &header_stream, &content_stream, HOST_TEST_PROXY, "http", HOST_TEST_PROXY, PATH_TEST_PROXY, headers );
	io_service.run();

	header = header_stream.str();
	content = content_stream.str();

	boost::to_lower ( content );
	boost::trim ( content );
	boost::erase_all ( content, "\r" );
	boost::erase_all ( content, "\n" );

	boost::regex ex;
	ex.assign ( "[\\s\\t]+" );

	std::ostringstream t(std::ios::out | std::ios::binary);
	std::ostream_iterator<char> oi(t);
	boost::regex_replace(oi, content.begin(), content.end(), ex, " ", boost::match_default | boost::format_all);
	content = t.str();
}

bool ProxyChecker::read_webpage( const string& server, const string& port, const string& host, const string path, HttpHeaderPairs& headers, ostream& header_stream, ostream& content_stream )
{
	cout << "Reading < " << "http://" << host << path << " > ... over < " << server << ":" << port << " > ... \n";

	boost::posix_time::ptime start_time = boost::posix_time::second_clock::local_time();
	int retry = 3;
	while ( retry -- )
	{
		try
		{
			read_try ( retry, server, port, host, path, headers, header_stream, content_stream );
			retry = 0;
			boost::posix_time::ptime end_time = boost::posix_time::second_clock::local_time();
			boost::posix_time::time_duration td = end_time - start_time;
			cout << " OK! (Use Time: " << boost::posix_time::to_simple_string ( td ) << ")\n";
			return true;
		}
		catch ( exception& ex )
		{
			if ( retry == 0 )
			{
				boost::posix_time::ptime end_time = boost::posix_time::second_clock::local_time();
				boost::posix_time::time_duration td = end_time - start_time;
				cout << " Failed ... (Use Time: " << boost::posix_time::to_simple_string ( td ) << ")\n";
				cerr << "\t< Exception Occured: " << ex.what() << " >\n";
			}
		}
	}

	return false;
}

void ProxyChecker::read_try ( int count, const string& server, const string& port, const string& host, const string path, HttpHeaderPairs& headers, ostream& header_stream, ostream& content_stream  )
{
	int times = 3 - count;
	if ( times == 1 )
	{
		cout << "  Trying ... the 1st time ...";
	}
	else if ( times == 2 )
	{
		cout << "  Trying ... the 2nd time ...";
	}
	else if ( times == 3 )
	{
		cout << "  Trying ... the 3rd time ...";
	}
	else
	{
		cout << "  Trying ... the " << times << "st time ...";
	}

	boost::asio::io_service io_service;
	WebpageHunter hunter ( io_service, &header_stream, &content_stream, server, port, host, path, headers, 5 );
	io_service.run();
}

bool ProxyChecker::verify_webpage ( const string& header, const string& content, stringstream& header_stream, stringstream& content_stream )
{
	string proxy_header, proxy_content;
	proxy_header = header_stream.str();
	proxy_content = content_stream.str();
	
	boost::to_lower ( proxy_content );
	boost::trim ( proxy_content );
	boost::erase_all ( proxy_content, "\r" );
	boost::erase_all ( proxy_content, "\n" );

	boost::regex ex;
	ex.assign ( "[\\s\\t]+" );

	std::ostringstream t(std::ios::out | std::ios::binary);
	std::ostream_iterator<char> oi(t);
	boost::regex_replace(oi, proxy_content.begin(), proxy_content.end(), ex, " ", boost::match_default | boost::format_all);
	proxy_content = t.str();

	if ( proxy_content == content )
	{
		return true;
	}
	else
	{
		return false;
	}
}